package com.hontlong.toy.snatcher;

import java.io.BufferedReader;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.OutputStreamWriter;
import java.io.PrintWriter;
import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;
import java.net.HttpURLConnection;
import java.net.MalformedURLException;
import java.net.SocketTimeoutException;
import java.net.URL;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.zip.GZIPInputStream;
import java.util.zip.Inflater;
import java.util.zip.InflaterInputStream;

import org.apache.log4j.Logger;

/**
 * 通过http协议下载网页内容
 * 
 * @author zhaigy
 * 
 */

public class HttpPageLoader {
	public enum HttpMethod {
		Get, Post
	}

	public static final String DEFAULT_CHARSET = "GBK";// "ISO-8859-1";

	/**
	 * Logger for this class
	 */
	private static final Logger logger = Logger.getLogger(HttpPageLoader.class);

	/**
	 * Lookup a character set name.
	 * <em>Vacuous for JVM's without <code>java.nio.charset</code>.</em>
	 * This uses reflection so the code will still run under prior JDK's but in
	 * that case the default is always returned.
	 * 
	 * @param name
	 *            The name to look up. One of the aliases for a character set.
	 * @param fallback
	 *            The name to return if the lookup fails.
	 * @return The character set name.
	 */
	public static String findCharset(final String name, final String fallback) {
		String ret;

		try {
			Class cls;
			Method method;
			Object object;

			cls = Class.forName("java.nio.charset.Charset");
			method = cls.getMethod("forName", new Class[] { String.class });
			object = method.invoke(null, new Object[] { name });
			method = cls.getMethod("name", new Class[] {});
			object = method.invoke(object, new Object[] {});
			ret = (String) object;
		} catch (final ClassNotFoundException cnfe) {
			// for reflection exceptions, assume the name is correct
			ret = name;
		} catch (final NoSuchMethodException nsme) {
			// for reflection exceptions, assume the name is correct
			ret = name;
		} catch (final IllegalAccessException ia) {
			// for reflection exceptions, assume the name is correct
			ret = name;
		} catch (final InvocationTargetException ita) {
			// java.nio.charset.IllegalCharsetNameException
			// and java.nio.charset.UnsupportedCharsetException
			// return the default
			ret = fallback;
			System.out
					.println("unable to determine cannonical charset name for "
							+ name + " - using " + fallback);
		}

		return (ret);
	}

	/**
	 * 分析内容并返回字符集标准名，如果可以找到字符集的话。
	 * 
	 * @param content
	 * @return 字符集名称或者null
	 */
	static public String getCharset(String content) {
		if (content == null) {
			return null;
		}
		content = content.toLowerCase();
		final String CHARSET_STRING = "charset";
		int index;
		String ret = null;

		// ret = DEFAULT_CHARSET;
		if (null != content) {
			index = content.indexOf(CHARSET_STRING);

			if (index != -1) {
				content = content.substring(index + CHARSET_STRING.length())
						.trim();
				if (content.startsWith("=")) {
					content = content.substring(1).trim();
					index = content.indexOf(";");
					if (index != -1) {
						content = content.substring(0, index);
					}

					// remove any double quotes from around charset string
					if (content.startsWith("\"") && content.endsWith("\"")
							&& (1 < content.length())) {
						content = content.substring(1, content.length() - 1);
					}

					// remove any single quote from around charset string
					if (content.startsWith("'") && content.endsWith("'")
							&& (1 < content.length())) {
						content = content.substring(1, content.length() - 1);
					}

					ret = HttpPageLoader.findCharset(content, ret);// 标准化的Charset名字

				}
			}
		}

		return (ret);
	}

	public static String getRequestHeader(final HttpURLConnection connection) {
		// dump it
		StringBuffer buffer;
		Map map;
		String key;
		List items;

		buffer = new StringBuffer(1024);
		buffer.append(connection.getRequestMethod());
		buffer.append(" ");
		buffer.append(connection.getURL());
		buffer.append(" HTTP/1.1\n");
		map = connection.getRequestProperties();
		for (final Iterator iter = map.keySet().iterator(); iter.hasNext();) {
			key = (String) iter.next();
			items = (List) map.get(key);
			buffer.append(key);
			buffer.append(": ");
			for (int i = 0; i < items.size(); i++) {
				if (0 != i) {
					buffer.append(", ");
				}
				buffer.append(items.get(i));
			}
			buffer.append("\n");
		}

		return (buffer.toString());
	}

	public static String getResponseHeader(final HttpURLConnection conn) {
		// dump it
		StringBuffer buffer;
		int code;
		String message;
		String key;
		String value;

		buffer = new StringBuffer(1024);
		try {
			code = conn.getResponseCode();
			if (-1 != code) {
				message = conn.getResponseMessage();
				for (int i = 0; null != (value = conn.getHeaderField(i)); i++) {
					key = conn.getHeaderFieldKey(i);
					if ((null == key) && (0 == i)) {
						buffer.append("HTTP/1.1 ");
						buffer.append(code);
						buffer.append(" ");
						buffer.append(message);
						buffer.append("\n");
					} else {
						if (null != key) {
							buffer.append(key);
							buffer.append(": ");
						}
						buffer.append(value);
						buffer.append("\n");
					}
				}
			}
		} catch (final IOException ioe) {
			buffer.append(ioe.toString());
		}

		return (buffer.toString());
	}

	/**
	 * @param string
	 * @return
	 */
	public static StringBuffer load(final String string) {
		final HttpPageLoader loader = new HttpPageLoader();
		return loader.download(string);
	}

	/**
	 * @uml.property name="method"
	 */
	private HttpMethod method;

	/**
	 * @uml.property name="params" multiplicity="(0 -1)" dimension="1"
	 */
	private String params;

	/**
	 * @uml.property name="url"
	 */
	private String urlString = null;

	private final Pattern patternCharset = Pattern.compile(
			"Content-Type[^<]+?content=[^<]+?charset=([\\w-]+)",
			Pattern.CASE_INSENSITIVE);

	public HttpPageLoader() {
	}

	public HttpPageLoader(final String url) {
		this.setUrlString(url);
	}

	/**
	 * HttpPageLoader
	 * 
	 * @param url
	 * @param method
	 * @param params
	 */
	public HttpPageLoader(final String url, final HttpMethod method,
			final String params) {
		this.setUrlString(url);
		this.setMethod(method);
		this.setParams(params);
	}

	/**
	 * download 下载页面，要求前面已经设置url、方法和参数
	 */
	public StringBuffer download() {
		if (HttpPageLoader.logger.isDebugEnabled()) {
			HttpPageLoader.logger.debug("download() - start"); //$NON-NLS-1$
		}
		if ((this.urlString == null) || (this.method == null)) {
			HttpPageLoader.logger.error("url地址或者提交方法没有设定");
			throw new NullPointerException("url地址或者提交方法没有设定");
		}

		String newUrlString = this.urlString;
		HttpURLConnection http = null;

		if (this.method == HttpMethod.Get) {
			if ((this.params != null) && (this.params.length() != 0)) {
				newUrlString = this.urlString + "?" + this.params;
			}
		}
		if (HttpPageLoader.logger.isDebugEnabled()) {
			HttpPageLoader.logger.debug("download() - 准备完毕，开始连接并下载"); //$NON-NLS-1$
			HttpPageLoader.logger.debug(this.method.toString() + " url:"
					+ newUrlString);
			if (this.method == HttpPageLoader.HttpMethod.Post) {
				HttpPageLoader.logger.debug("POST PARAMS:" + this.params);
			}
		}

		URL url = null;
		StringBuffer buffer = null;
		BufferedReader in = null;
		try {
			url = new URL(newUrlString);

			http = (HttpURLConnection) url.openConnection();
			http.setConnectTimeout(60 * 1000);// 60m
			http.setReadTimeout(60 * 1000);// 60m

			// http.setInstanceFollowRedirects(false);

			http.setRequestProperty("Accept",
					"text/html;q=0.9,text/plain;q=0.8,*/*;q=0.5");
			http.setRequestProperty("Accept-Language", "zh-cn,zh;q=0.5");
			http.setRequestProperty("Accept-Encoding", "gzip, deflate");
			http.setRequestProperty("Accept-Charset",
					"gbk,gb2312,utf-8;q=0.7,*;q=0.7");
			http.setRequestProperty("User-Agent",
					"Mozilla/4.0 (compatible; MSIE 5.5; Windows NT 5.0)");
			http.setRequestProperty("Connection", "Keep-Alive");
			http.setRequestProperty("Cookie", "");

			if (newUrlString.indexOf(".job88.") >= 0) {// fro job88
				http.setRequestProperty("x-ajaxpro-method", "GetResult");
			}
			if (this.urlString.indexOf("job168") >= 0) {// for job168
				Thread.sleep(1500);
			}
			// System.out.println(this.params);

			if (this.method == HttpPageLoader.HttpMethod.Post) {
				http.setRequestMethod("POST");
				http.setDoOutput(true);
				final PrintWriter out = new PrintWriter(new OutputStreamWriter(
						http.getOutputStream()));
				// System.out.println(newUrlString+"?"+this.params);
				out.println(this.params);
				out.close();
			} else if (this.method == HttpMethod.Get) {
				http.connect();
				// System.out.println(newUrlString);
			}

			// Map<String,List<String>> headMap=http.getHeaderFields();
			// for(Entry<String,List<String>> e:headMap.entrySet()){
			// System.out.println(e.getKey());
			// }
			//       

			if (HttpPageLoader.logger.isDebugEnabled()) {
				HttpPageLoader.logger.debug(HttpPageLoader
						.getResponseHeader(http));
			}
			// int code = http.getResponseCode();
			// if (3 == (code / 100)) {
			// logger.info("转连接：ResponseCode=" + code);
			// // TO DO 转连接处理
			// // HTTP/1.1 302 Found
			// // Date: Tue, 30 Jan 2007 04:57:06 GMT
			// // Server: Microsoft-IIS/6.0
			// // X-Powered-By: ASP.NET
			// // X-AspNet-Version: 1.1.4322
			// // Location: /Index.aspx
			// // Set-Cookie: ASP.NET_SessionId=hk14sd45qwukp0etuqqmk3e1;
			// // path=/
			// // Cache-Control: private
			// // Content-Type: text/html; charset=gb2312
			// // Content-Length: 128
			// }
			if (http.getResponseCode() != HttpURLConnection.HTTP_OK) {
				throw new RuntimeException("连接服务器下载失败:ResponseCode="
						+ http.getResponseCode());
			}
			final String urlLink = http.getURL().toString();
			// 处理ChinaHr招聘会
			// http://pages.chinahr.com/templet/default.asp?ContractID=20070322023814&single_service_id=160556
			if (urlLink.toLowerCase().matches(".*chinahr.*\\?contractid=.*")
					&& !urlLink.equals(newUrlString)) {
				HttpPageLoader.logger.info("原始链接:" + newUrlString);
				HttpPageLoader.logger.info("ChinaHr重定向链接:" + urlLink);
				return null;
			}
			if (HttpPageLoader.logger.isDebugEnabled()) {
				HttpPageLoader.logger.debug("download() - 连接成功"); //$NON-NLS-1$
			}
			// 读取网页的编码类型并设置输入流的编码器
			// TODO 页面的编码器使用下面的方式获取是不可靠的，要么自动探测，要么从配置中传入！！！
			// String webContentType = http.getContentType();
			//
			// String webCharSet = getCharset(webContentType);

			// if (urlString.toLowerCase().contains("zhaopin")) {
			// webCharSet = "UTF-8";
			// }
			// if (webCharSet.equalsIgnoreCase("GB2312")) {
			// webCharSet = "GBK";
			// }

			InputStream stream = null;
			final String contentEncoding = http.getContentEncoding();
			if ((null != contentEncoding)
					&& (-1 != contentEncoding.indexOf("gzip"))) {
				stream = new GZIPInputStream(http.getInputStream());
			} else if ((null != contentEncoding)
					&& (-1 != contentEncoding.indexOf("deflate"))) {
				stream = new InflaterInputStream(http.getInputStream(),
						new Inflater(true));
			} else {
				stream = http.getInputStream();
			}

			final String webContentType = http.getContentType();
			final int MAX_LEN_PAGE = 1024 * 1024;// 1000k
			final ByteArrayOutputStream outbuff = new ByteArrayOutputStream(
					200 * 1024);
			{
				final byte[] bs = new byte[64 * 1024];
				int alllen = 0;
				for (int len; (len = stream.read(bs)) > 0;) {
					alllen += len;
					if (alllen > MAX_LEN_PAGE) {
						break;
					}
					outbuff.write(bs, 0, len);
				}
			}
			stream.close();
			final byte[] bs = outbuff.toByteArray();
			outbuff.close();
			stream = new ByteArrayInputStream(bs);

			String webCharSet = HttpPageLoader.getCharset(webContentType);
			if (webCharSet == null) {
				// 使用编码头信息
				// 检测文件头标识，提示用户选择，根据一定的规则猜测
				// 最标准的途径是检测文本最开头的几个字节，开头字节 Charset/encoding,如下表：
				// EF BB BF UTF-8
				// FE FF UTF-16/UCS-2, little endian
				// FF FE UTF-16/UCS-2, big endian
				// FF FE 00 00 UTF-32/UCS-4, little endian.
				// 00 00 FE FF UTF-32/UCS-4, big-endian.
				if (bs.length > 2) {
					if (bs[0] == (byte) 0xef) {
						if (bs[1] == (byte) 0xbb) {
							webCharSet = "UTF-8";
						}
					} else {

					}
				}
				// 使用内容判断
				if (webCharSet == null) {
					final BufferedReader temp = new BufferedReader(
							new InputStreamReader(stream, "ISO-8859-1"));
					// <meta http-equiv="Content-Type" content="text/html;
					// charset=gb2312" />
					int top = 0;
					for (String line; (line = temp.readLine()) != null;) {
						// <?xml version="1.0" encoding="gb2312" ?>
						// for xhtml
						if (top < 5) {// 只处理前5行
							if (line.contains("<?xml")) {
								final int idx = line.indexOf("encoding=\"");
								if (idx > 0) {
									final int idx2 = line.indexOf("\"", idx
											+ "encoding=\"".length());
									if (idx2 > 0) {
										final String sub = line.substring(idx
												+ "encoding=\"".length(), idx2);
										webCharSet = sub;
										break;
									}
								}
							}
							top++;
						}
						final Matcher m = this.patternCharset.matcher(line);
						if (m.find()) {
							if (m.groupCount() > 0) {
								webCharSet = m.group(1);
								break;
							}
						}
					}
					temp.close();
				}
				if (webCharSet == null) {
					webCharSet = HttpPageLoader.DEFAULT_CHARSET;
				}
			}

			webCharSet = HttpPageLoader.findCharset(webCharSet,
					HttpPageLoader.DEFAULT_CHARSET);

			if (webCharSet.indexOf("GB2312") >= 0) {// GB2312->GBK
				webCharSet = HttpPageLoader.DEFAULT_CHARSET;
			}
			stream.reset();
			in = new BufferedReader(new InputStreamReader(stream, webCharSet));

			String line = null;
			buffer = new StringBuffer();
			if (HttpPageLoader.logger.isDebugEnabled()) {
				HttpPageLoader.logger.debug("download() - 开始下载"); //$NON-NLS-1$
			}
			buffer.append("<!--url=" + this.urlString + "-->\n");
			buffer.append("<!--params=" + this.params + "-->\n");
			buffer.append("<!--method=" + this.method + "-->\n");
			buffer.append("<!--charset=" + webCharSet + "-->\n");
			while ((line = in.readLine()) != null) {
				buffer.append(line).append('\n');
			}
			if (HttpPageLoader.logger.isDebugEnabled()) {
				HttpPageLoader.logger.debug("download() - 下载成功"); //$NON-NLS-1$
			}
		} catch (final SocketTimeoutException e) {
			HttpPageLoader.logger.error("download()", e); //$NON-NLS-1$
		} catch (final MalformedURLException e) {
			HttpPageLoader.logger.error("download()", e); //$NON-NLS-1$
		} catch (final IOException e) {
			HttpPageLoader.logger.error("download()", e); //$NON-NLS-1$
		} catch (final Exception e) {
			HttpPageLoader.logger.error("download()", e); //$NON-NLS-1$
		} finally {
			try {
				if (in != null) {
					in.close();
				}
			} catch (final IOException e) {
				e.printStackTrace();
			}
			if (http != null) {
				http.disconnect();
			}

		}
		if (HttpPageLoader.logger.isDebugEnabled()) {
			HttpPageLoader.logger.debug("download() - end"); //$NON-NLS-1$
		}
		return buffer;

	}

	/**
	 * download 下载页面，要求前面已经设置url、方法和参数
	 */
	public StringBuffer download(final String downLoadUrl) {
		if (HttpPageLoader.logger.isDebugEnabled()) {
			HttpPageLoader.logger.debug("download() - start"); //$NON-NLS-1$
		}
		this.setUrlString(downLoadUrl);
		if (this.method == null) {
			this.setMethod(HttpMethod.Get);
		}
		return this.download();
	}

	/**
	 * @uml.property name="method"
	 */

	public HttpMethod getMethod() {
		return this.method;
	}

	/**
	 * @uml.property name="params"
	 */
	public String getParams() {
		return this.params;
	}

	/**
	 * @uml.property name="url"
	 */
	public String getUrlString() {
		return this.urlString;
	}

	/**
	 * @uml.property name="method"
	 */
	public void setMethod(final HttpMethod method) {
		this.method = method;
	}

	/**
	 * @uml.property name="params"
	 */
	public void setParams(final String params) {
		this.params = params;
	}

	/**
	 * @uml.property name="url"
	 */
	public void setUrlString(final String url) {
		this.urlString = url;
	}
}
